#! /usr/bin/python
#  -*- coding: utf-8 -*-
import Queue
import threading
import time
import urllib2

from BeautifulSoup import BeautifulSoup
from libs import htmlparser
from libs import sqliteDB
from libs.functions import *
import stackless
class geturls:
    def __init__(self, db):
        self.db = db


    def getUrls(self, num):
        sql = "select urls ,id from urls order by nums desc limit %d" % num
        
        res = self.db.fetch(sql, 2)
        self.db.commit()
        r = []
        ids = []
        if res:
            
            for v in  res:
                r.append(v['urls'])
                ids.append(str(v['id']))
            t = str(int(time.time()))
            sql = "update urls set nums=%s where id in (%s)" % (t, ','.join(ids))
            self.db.execsql(sql)
            self.db.commit()
            
           
            
            return r
        else:
            return ['http://shop.0791look.com/', ]


    def setUrls(self, urls):
        

        sql = "insert into urls (urls) values ('%s')" % urls[0]
        self.db.execsql(sql)
        self.db.commit()
        
    def setXml(self, urls):
        self.db.execsql("insert into xmls (urls) values (?)", urls)
        self.db.commit()


def sqler(tt):
    while channel3.receive():
           
        global DB
        #stackless.schedule()
        
        sql = sqlquene.get()
        print sql
        try:
            DB.execsql(sql)
            DB.commit()
        except:
            pass
   

def urlsreturn():

    
    while channel2.receive():
        global urler
        url = urler.getUrls(5)
        print url
        print 'kk'
        for u in url:
            
            urlqueu.put(u)
            print urlqueu.qsize()
        channel1.send('1')
        channel3.send('1')



def do_s():
  
        

    
    while channel1.receive():
       
        url = urlqueu.get()
        print 'ppp'
        channel2.send(3)
        urlpath = hostParser(url)
        try:
            uh = urllib2.urlopen(url)
            html = uh.read()
            htmers = htmlparser.htmler(html)
            arr = htmers.getTags('a')
            j = 1
            for i in arr:
                try:
                    u = i['href']
                    if u[0:7] == 'http://'  and   u.find(urlpath['superhost']) > -1:

                        urler.setUrls([u])


                except:
                    pass
        except:
            print "ERR0:%s" % url




   



    
        


if __name__ == '__main__':
    channel1 = stackless.channel()
    channel2 = stackless.channel()
    channel3 = stackless.channel()
    DB = sqliteDB. sqlite_db('data/db.s3db')
    urler = geturls(DB)
    sqlquene = Queue.Queue()
    urlqueu = Queue.Queue()
    stackless.tasklet(urlsreturn)()
    stackless.tasklet(do_s)()
    stackless.tasklet(sqler)('kk')
    stackless.tasklet(channel2.send)(1)
    stackless.run()
    


    

        
        
    

    
    
    

